package com.hlmc.spider;

import org.springframework.stereotype.Service;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;

@Service("fl_spider")
public class FuLi_spider extends BaseProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setCharset("utf-8" + "");

    @Override
    public void process(Page page) {



        if(page.getUrl().toString().contains("https://www.520szy.com/lsnzy/?s=Home-vod-show-id-1")){
            page.addTargetRequests(page.getHtml().xpath("//ul[@class='videoContent']/li/a[1]/@href").replace(" ","").all());
            page.addTargetRequests(page.getHtml().xpath("//a[@class='next pagegbk']/@href").all());

        }
        if (page.getUrl().toString().contains("https://www.520szy.com/lsnzy/?s=Home-vod-read-")){
            String title=page.getHtml().xpath("//div[@class='list-content']/h4/text()").toString().replace("影片名称：","");
            String area=page.getHtml().xpath("//div[@class='new_right']/p[2]/text()").toString();
            String toup=page.getHtml().xpath("//div[@class='new_right']/p[5]/text()").toString();
            String url=page.getHtml().xpath("//div[@class='play-tool']/div[4]/p/a/text()").toString();

            if( (url == null)||url.replace(" ", "").equals("")) {

                page.setSkip(true);
            }
            System.out.println(title);
            System.out.println(url);

            page.putField("title", title);
            page.putField("toup", toup);
            page.putField("area", area);
            page.putField("url", url);

        }

    }

    @Override
    public Site getSite() {
        return site;
    }


    public static void main(String[] args){
        Spider.create(new FuLi_spider())
                .addUrl("https://www.520szy.com/lsnzy/?s=Home-vod-show-id-1.html")
                .thread(5)
                .run();

    }

}

